from pyspark import SparkConf, SparkContext

if __name__ == '__main__':
    conf = SparkConf().setAppName("test").setMaster("local[*]")
    sc = SparkContext(conf=conf)

    rdd = sc.parallelize(["hadoop spark hadoop", "spark hadoop hadoop", "hadoop flink spark"])
    rdd2 = rdd.flatMap(lambda x: x.split(" "))
    rdd3 = rdd2.map(lambda x: (x, 1))

    # groupBy传入的函数 意思是：通过这个函数，确定按照谁来分组
    result = rdd3.groupBy(lambda t: t[0])

    # [('spark', [('spark', 1), ('spark', 1), ('spark', 1)]), ('hadoop', [('hadoop', 1), ('hadoop', 1), ('hadoop', 1), ('hadoop', 1), ('hadoop', 1)]), ('flink', [('flink', 1)])]
    print(result.map(lambda t: (t[0], list(t[1]))).collect())
